{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "PyTorch的Tensor\n",
    "----------------\n",
    "\n",
    "和前面一样，我们还是实现一个全连接的Relu激活的网络，它只有一个隐层并且没有bias。loss是预测与真实值的欧氏距离。\n",
    "\n",
    "\n",
    "之前我们用Numpy实现，自己手动前向计算loss，反向计算梯度。这里还是一样，只不过把numpy数组换成了PyTorch的Tensor。\n",
    "\n",
    "但是使用PyTorch的好处是我们可以利用GPU来加速计算，如果想用GPU计算，我们值需要在创建tensor的时候指定device为gpu。\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 27160548.0\n",
      "1 23449182.0\n",
      "2 26120808.0\n",
      "3 32149890.0\n",
      "4 37190440.0\n",
      "5 35832544.0\n",
      "6 26425754.0\n",
      "7 14859924.0\n",
      "8 6947733.0\n",
      "9 3219258.5\n",
      "10 1716543.5\n",
      "11 1111947.25\n",
      "12 835275.8125\n",
      "13 681324.5\n",
      "14 578198.625\n",
      "15 500179.78125\n",
      "16 437220.53125\n",
      "17 384775.625\n",
      "18 340300.59375\n",
      "19 302186.21875\n",
      "20 269241.375\n",
      "21 240663.921875\n",
      "22 215757.671875\n",
      "23 193945.390625\n",
      "24 174762.15625\n",
      "25 157839.625\n",
      "26 142858.71875\n",
      "27 129555.2890625\n",
      "28 117708.453125\n",
      "29 107117.5234375\n",
      "30 97642.5078125\n",
      "31 89144.1484375\n",
      "32 81506.3515625\n",
      "33 74628.2109375\n",
      "34 68425.6328125\n",
      "35 62819.69140625\n",
      "36 57745.23046875\n",
      "37 53145.54296875\n",
      "38 48966.77734375\n",
      "39 45171.5234375\n",
      "40 41716.015625\n",
      "41 38566.4140625\n",
      "42 35691.43359375\n",
      "43 33060.91796875\n",
      "44 30652.283203125\n",
      "45 28444.80078125\n",
      "46 26417.853515625\n",
      "47 24555.509765625\n",
      "48 22842.537109375\n",
      "49 21265.677734375\n",
      "50 19814.068359375\n",
      "51 18478.091796875\n",
      "52 17244.171875\n",
      "53 16104.5859375\n",
      "54 15050.564453125\n",
      "55 14074.8291015625\n",
      "56 13172.9384765625\n",
      "57 12337.578125\n",
      "58 11562.255859375\n",
      "59 10842.37109375\n",
      "60 10173.2822265625\n",
      "61 9550.9853515625\n",
      "62 8971.9658203125\n",
      "63 8432.5712890625\n",
      "64 7930.03466796875\n",
      "65 7461.08251953125\n",
      "66 7023.52587890625\n",
      "67 6614.81396484375\n",
      "68 6233.02587890625\n",
      "69 5876.06298828125\n",
      "70 5542.21923828125\n",
      "71 5230.03857421875\n",
      "72 4937.73095703125\n",
      "73 4663.67626953125\n",
      "74 4406.6806640625\n",
      "75 4165.6455078125\n",
      "76 3939.5029296875\n",
      "77 3727.212890625\n",
      "78 3527.802734375\n",
      "79 3340.528564453125\n",
      "80 3164.386474609375\n",
      "81 2998.921875\n",
      "82 2842.981201171875\n",
      "83 2696.11328125\n",
      "84 2557.7021484375\n",
      "85 2427.264404296875\n",
      "86 2304.234619140625\n",
      "87 2188.219482421875\n",
      "88 2078.69873046875\n",
      "89 1975.29931640625\n",
      "90 1877.6087646484375\n",
      "91 1785.4027099609375\n",
      "92 1698.1796875\n",
      "93 1615.6761474609375\n",
      "94 1537.605712890625\n",
      "95 1463.7264404296875\n",
      "96 1393.8037109375\n",
      "97 1327.58935546875\n",
      "98 1264.8487548828125\n",
      "99 1205.3917236328125\n",
      "100 1149.02099609375\n",
      "101 1095.5726318359375\n",
      "102 1044.8736572265625\n",
      "103 996.8003540039062\n",
      "104 951.1334838867188\n",
      "105 907.7781372070312\n",
      "106 866.6013793945312\n",
      "107 827.4901123046875\n",
      "108 790.3089599609375\n",
      "109 754.9550170898438\n",
      "110 721.3450927734375\n",
      "111 689.391845703125\n",
      "112 658.9813232421875\n",
      "113 630.0376586914062\n",
      "114 602.486328125\n",
      "115 576.2611083984375\n",
      "116 551.2786865234375\n",
      "117 527.5062866210938\n",
      "118 504.826904296875\n",
      "119 483.2142639160156\n",
      "120 462.6153259277344\n",
      "121 442.978515625\n",
      "122 424.2507019042969\n",
      "123 406.395751953125\n",
      "124 389.3479919433594\n",
      "125 373.0765380859375\n",
      "126 357.5439453125\n",
      "127 342.7160339355469\n",
      "128 328.5504150390625\n",
      "129 315.0167541503906\n",
      "130 302.0850524902344\n",
      "131 289.73089599609375\n",
      "132 277.91839599609375\n",
      "133 266.6354064941406\n",
      "134 255.84048461914062\n",
      "135 245.5149688720703\n",
      "136 235.63674926757812\n",
      "137 226.18853759765625\n",
      "138 217.14523315429688\n",
      "139 208.4910430908203\n",
      "140 200.20436096191406\n",
      "141 192.2769317626953\n",
      "142 184.68177795410156\n",
      "143 177.40785217285156\n",
      "144 170.4374237060547\n",
      "145 163.76333618164062\n",
      "146 157.3675079345703\n",
      "147 151.2388458251953\n",
      "148 145.36354064941406\n",
      "149 139.73255920410156\n",
      "150 134.334716796875\n",
      "151 129.15769958496094\n",
      "152 124.1950454711914\n",
      "153 119.43408966064453\n",
      "154 114.86677551269531\n",
      "155 110.48383331298828\n",
      "156 106.27606201171875\n",
      "157 102.24146270751953\n",
      "158 98.36918640136719\n",
      "159 94.65274810791016\n",
      "160 91.08273315429688\n",
      "161 87.6547622680664\n",
      "162 84.36548614501953\n",
      "163 81.20487213134766\n",
      "164 78.1706771850586\n",
      "165 75.25331115722656\n",
      "166 72.45304870605469\n",
      "167 69.76226806640625\n",
      "168 67.17477416992188\n",
      "169 64.68970489501953\n",
      "170 62.30183410644531\n",
      "171 60.007606506347656\n",
      "172 57.79960250854492\n",
      "173 55.67739486694336\n",
      "174 53.63734436035156\n",
      "175 51.677734375\n",
      "176 49.79066467285156\n",
      "177 47.9757194519043\n",
      "178 46.230323791503906\n",
      "179 44.552616119384766\n",
      "180 42.93801498413086\n",
      "181 41.383148193359375\n",
      "182 39.88833999633789\n",
      "183 38.44978332519531\n",
      "184 37.06580352783203\n",
      "185 35.73257064819336\n",
      "186 34.44986343383789\n",
      "187 33.215877532958984\n",
      "188 32.02786636352539\n",
      "189 30.8831844329834\n",
      "190 29.781312942504883\n",
      "191 28.720050811767578\n",
      "192 27.698816299438477\n",
      "193 26.715251922607422\n",
      "194 25.76749610900879\n",
      "195 24.855438232421875\n",
      "196 23.975658416748047\n",
      "197 23.129318237304688\n",
      "198 22.313077926635742\n",
      "199 21.527206420898438\n",
      "200 20.76997184753418\n",
      "201 20.04037094116211\n",
      "202 19.336854934692383\n",
      "203 18.660053253173828\n",
      "204 18.006799697875977\n",
      "205 17.37683868408203\n",
      "206 16.770252227783203\n",
      "207 16.18552589416504\n",
      "208 15.622089385986328\n",
      "209 15.078423500061035\n",
      "210 14.554393768310547\n",
      "211 14.049172401428223\n",
      "212 13.56241226196289\n",
      "213 13.092257499694824\n",
      "214 12.63919448852539\n",
      "215 12.202266693115234\n",
      "216 11.781222343444824\n",
      "217 11.374651908874512\n",
      "218 10.98249626159668\n",
      "219 10.604256629943848\n",
      "220 10.239360809326172\n",
      "221 9.887673377990723\n",
      "222 9.548309326171875\n",
      "223 9.220595359802246\n",
      "224 8.904598236083984\n",
      "225 8.599783897399902\n",
      "226 8.305654525756836\n",
      "227 8.021685600280762\n",
      "228 7.748071193695068\n",
      "229 7.483476638793945\n",
      "230 7.22836971282959\n",
      "231 6.982523441314697\n",
      "232 6.744885444641113\n",
      "233 6.515468120574951\n",
      "234 6.293992042541504\n",
      "235 6.08054256439209\n",
      "236 5.8743157386779785\n",
      "237 5.675258159637451\n",
      "238 5.4830451011657715\n",
      "239 5.2976837158203125\n",
      "240 5.118594646453857\n",
      "241 4.945697784423828\n",
      "242 4.7789812088012695\n",
      "243 4.61752462387085\n",
      "244 4.462174415588379\n",
      "245 4.31167459487915\n",
      "246 4.166831970214844\n",
      "247 4.026605606079102\n",
      "248 3.8910975456237793\n",
      "249 3.7603092193603516\n",
      "250 3.6340250968933105\n",
      "251 3.512382745742798\n",
      "252 3.394530773162842\n",
      "253 3.280857801437378\n",
      "254 3.171160936355591\n",
      "255 3.065077304840088\n",
      "256 2.9626712799072266\n",
      "257 2.8636186122894287\n",
      "258 2.7680463790893555\n",
      "259 2.6756556034088135\n",
      "260 2.586381435394287\n",
      "261 2.5001213550567627\n",
      "262 2.416795253753662\n",
      "263 2.3365061283111572\n",
      "264 2.258803129196167\n",
      "265 2.183699607849121\n",
      "266 2.111043691635132\n",
      "267 2.0409533977508545\n",
      "268 1.973219633102417\n",
      "269 1.9077643156051636\n",
      "270 1.8445075750350952\n",
      "271 1.7833516597747803\n",
      "272 1.7241637706756592\n",
      "273 1.6671454906463623\n",
      "274 1.6119701862335205\n",
      "275 1.5587425231933594\n",
      "276 1.5072194337844849\n",
      "277 1.4573854207992554\n",
      "278 1.409378170967102\n",
      "279 1.3627963066101074\n",
      "280 1.3178181648254395\n",
      "281 1.2744096517562866\n",
      "282 1.232387661933899\n",
      "283 1.1916441917419434\n",
      "284 1.1524087190628052\n",
      "285 1.114516258239746\n",
      "286 1.0778025388717651\n",
      "287 1.042418360710144\n",
      "288 1.0081615447998047\n",
      "289 0.9750333428382874\n",
      "290 0.942963182926178\n",
      "291 0.9120372533798218\n",
      "292 0.8821767568588257\n",
      "293 0.8532135486602783\n",
      "294 0.825335681438446\n",
      "295 0.7981885075569153\n",
      "296 0.7720930576324463\n",
      "297 0.7467435002326965\n",
      "298 0.7223318219184875\n",
      "299 0.698689341545105\n",
      "300 0.675778865814209\n",
      "301 0.6537696123123169\n",
      "302 0.6323790550231934\n",
      "303 0.6117067933082581\n",
      "304 0.5917704105377197\n",
      "305 0.5724595189094543\n",
      "306 0.5537658929824829\n",
      "307 0.5356149673461914\n",
      "308 0.5182029604911804\n",
      "309 0.5012991428375244\n",
      "310 0.48496493697166443\n",
      "311 0.4691324830055237\n",
      "312 0.45386192202568054\n",
      "313 0.43907320499420166\n",
      "314 0.4247817099094391\n",
      "315 0.4109227657318115\n",
      "316 0.3975951671600342\n",
      "317 0.3846861720085144\n",
      "318 0.3721347451210022\n",
      "319 0.36009085178375244\n",
      "320 0.34836557507514954\n",
      "321 0.3370331823825836\n",
      "322 0.3260921835899353\n",
      "323 0.3155136704444885\n",
      "324 0.30524101853370667\n",
      "325 0.29540687799453735\n",
      "326 0.2858385145664215\n",
      "327 0.27654531598091125\n",
      "328 0.26755475997924805\n",
      "329 0.25887903571128845\n",
      "330 0.2505364716053009\n",
      "331 0.2424183040857315\n",
      "332 0.23460844159126282\n",
      "333 0.22700633108615875\n",
      "334 0.21966645121574402\n",
      "335 0.21255823969841003\n",
      "336 0.2056957334280014\n",
      "337 0.19903601706027985\n",
      "338 0.19257794320583344\n",
      "339 0.18637560307979584\n",
      "340 0.18039099872112274\n",
      "341 0.174573615193367\n",
      "342 0.16893401741981506\n",
      "343 0.16349799931049347\n",
      "344 0.1582200825214386\n",
      "345 0.15310266613960266\n",
      "346 0.14818622171878815\n",
      "347 0.14339964091777802\n",
      "348 0.13876791298389435\n",
      "349 0.13428270816802979\n",
      "350 0.1299685537815094\n",
      "351 0.12577421963214874\n",
      "352 0.12175603955984116\n",
      "353 0.11781707406044006\n",
      "354 0.11402475833892822\n",
      "355 0.11035911738872528\n",
      "356 0.1068304106593132\n",
      "357 0.10336054116487503\n",
      "358 0.10005467385053635\n",
      "359 0.09681405872106552\n",
      "360 0.09374134242534637\n",
      "361 0.09073787182569504\n",
      "362 0.08782388269901276\n",
      "363 0.08498760312795639\n",
      "364 0.08226948231458664\n",
      "365 0.07963994145393372\n",
      "366 0.07708512246608734\n",
      "367 0.07460156828165054\n",
      "368 0.07223011553287506\n",
      "369 0.0699114128947258\n",
      "370 0.06766950339078903\n",
      "371 0.06550011038780212\n",
      "372 0.06341234594583511\n",
      "373 0.061372723430395126\n",
      "374 0.05939136818051338\n",
      "375 0.05751316249370575\n",
      "376 0.055672526359558105\n",
      "377 0.05389607697725296\n",
      "378 0.05216585472226143\n",
      "379 0.05050181224942207\n",
      "380 0.048898208886384964\n",
      "381 0.04733841493725777\n",
      "382 0.04584401473402977\n",
      "383 0.04436405375599861\n",
      "384 0.04295770451426506\n",
      "385 0.04158620536327362\n",
      "386 0.04027438536286354\n",
      "387 0.038986653089523315\n",
      "388 0.037750404328107834\n",
      "389 0.03654330223798752\n",
      "390 0.03538602963089943\n",
      "391 0.03426051512360573\n",
      "392 0.03318720683455467\n",
      "393 0.03212898224592209\n",
      "394 0.03110460564494133\n",
      "395 0.030130209401249886\n",
      "396 0.02917947992682457\n",
      "397 0.028250863775610924\n",
      "398 0.027351628988981247\n",
      "399 0.026493802666664124\n",
      "400 0.025651264935731888\n",
      "401 0.024842334911227226\n",
      "402 0.024061692878603935\n",
      "403 0.02330697514116764\n",
      "404 0.022578416392207146\n",
      "405 0.021867625415325165\n",
      "406 0.021188633516430855\n",
      "407 0.02052735909819603\n",
      "408 0.019880136474967003\n",
      "409 0.019251517951488495\n",
      "410 0.018648866564035416\n",
      "411 0.018060481175780296\n",
      "412 0.017501454800367355\n",
      "413 0.016960710287094116\n",
      "414 0.016431933268904686\n",
      "415 0.015926344320178032\n",
      "416 0.015425550751388073\n",
      "417 0.014946338720619678\n",
      "418 0.014481154270470142\n",
      "419 0.014031954109668732\n",
      "420 0.013592609204351902\n",
      "421 0.013181440532207489\n",
      "422 0.012772885151207447\n",
      "423 0.012371894903481007\n",
      "424 0.011992516927421093\n",
      "425 0.011626292020082474\n",
      "426 0.011271747760474682\n",
      "427 0.010925785638391972\n",
      "428 0.010592750273644924\n",
      "429 0.010269062593579292\n",
      "430 0.009953935630619526\n",
      "431 0.009652292355895042\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "432 0.00935486238449812\n",
      "433 0.00907206628471613\n",
      "434 0.008797461166977882\n",
      "435 0.008530347608029842\n",
      "436 0.008267566561698914\n",
      "437 0.008022594265639782\n",
      "438 0.007777847815304995\n",
      "439 0.007544893771409988\n",
      "440 0.007315889932215214\n",
      "441 0.007104233838617802\n",
      "442 0.006887681782245636\n",
      "443 0.006679466459900141\n",
      "444 0.006482238415628672\n",
      "445 0.006291826721280813\n",
      "446 0.006104573607444763\n",
      "447 0.005923294462263584\n",
      "448 0.005745504982769489\n",
      "449 0.005578520707786083\n",
      "450 0.005415928550064564\n",
      "451 0.005256837699562311\n",
      "452 0.005104752257466316\n",
      "453 0.004957721568644047\n",
      "454 0.004813564009964466\n",
      "455 0.004674824420362711\n",
      "456 0.00453700078651309\n",
      "457 0.004409389570355415\n",
      "458 0.004281625151634216\n",
      "459 0.004159301985055208\n",
      "460 0.00403966661542654\n",
      "461 0.003928312100470066\n",
      "462 0.0038173189386725426\n",
      "463 0.003707451745867729\n",
      "464 0.003601861884817481\n",
      "465 0.003503216663375497\n",
      "466 0.003407241078093648\n",
      "467 0.0033099823631346226\n",
      "468 0.003218397730961442\n",
      "469 0.003128425218164921\n",
      "470 0.0030428029131144285\n",
      "471 0.00296075944788754\n",
      "472 0.002881790976971388\n",
      "473 0.0028022180777043104\n",
      "474 0.002724785590544343\n",
      "475 0.002650971757248044\n",
      "476 0.0025823393370956182\n",
      "477 0.0025146123953163624\n",
      "478 0.0024463904555886984\n",
      "479 0.002381637692451477\n",
      "480 0.002321274485439062\n",
      "481 0.002261445391923189\n",
      "482 0.0022017944138497114\n",
      "483 0.0021446640603244305\n",
      "484 0.0020902554970234632\n",
      "485 0.0020351125858724117\n",
      "486 0.001982203684747219\n",
      "487 0.0019321513827890158\n",
      "488 0.0018812519265338778\n",
      "489 0.0018363372655585408\n",
      "490 0.0017893637996166945\n",
      "491 0.0017444592667743564\n",
      "492 0.0017008378636091948\n",
      "493 0.0016584914410486817\n",
      "494 0.0016177300130948424\n",
      "495 0.0015782773261889815\n",
      "496 0.0015395194059237838\n",
      "497 0.0015024702297523618\n",
      "498 0.0014665626222267747\n",
      "499 0.0014313171850517392\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "\n",
    "dtype = torch.float\n",
    "device = torch.device(\"cpu\")\n",
    "# device = torch.device(\"cuda:0\") # 如果想在GPU上运算，把这行注释掉。\n",
    " \n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    " \n",
    "x = torch.randn(N, D_in, device=device, dtype=dtype)\n",
    "y = torch.randn(N, D_out, device=device, dtype=dtype)\n",
    " \n",
    "w1 = torch.randn(D_in, H, device=device, dtype=dtype)\n",
    "w2 = torch.randn(H, D_out, device=device, dtype=dtype)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "for t in range(500): \n",
    "    h = x.mm(w1)\n",
    "    h_relu = h.clamp(min=0)\n",
    "    y_pred = h_relu.mm(w2)\n",
    " \n",
    "    loss = (y_pred - y).pow(2).sum().item()\n",
    "    print(t, loss)\n",
    " \n",
    "    grad_y_pred = 2.0 * (y_pred - y)\n",
    "    grad_w2 = h_relu.t().mm(grad_y_pred)\n",
    "    grad_h_relu = grad_y_pred.mm(w2.t())\n",
    "    grad_h = grad_h_relu.clone()\n",
    "    grad_h[h < 0] = 0\n",
    "    grad_w1 = x.t().mm(grad_h)\n",
    " \n",
    "    w1 -= learning_rate * grad_w1\n",
    "    w2 -= learning_rate * grad_w2"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py3.6-env",
   "language": "python",
   "name": "py3.6-env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
